missing = []
there = []
for id in obsids:
    try:
        data = UVPDS(id, skip_download=True)
    except FileNotFoundError:
        print(id, "not there.")
        missing.append(id)
    else:
        print("Got", id)
        there.append(id)

---------------------------------------------------------------------------
NameError                                 Traceback (most recent call last)
/tmp/ipykernel_143809/2093082731.py in <module>
      1 missing = []
      2 there = []
----> 3 for id in obsids:
      4     try:
      5         data = UVPDS(id, skip_download=True)

NameError: name 'obsids' is not defined

['2001-093', '2002-198', '2003-139']

cat = CatalogFilter(steffl_spica_dates[2])

Stored index is up-to-date.

cat.date = "2003-05-19"

pids = list(cat.get_euv_date().query("OBSERVATION_TYPE=='CALIB'").index)

pids

['EUV2003_139_18_11',
 'EUV2003_139_19_07',
 'EUV2003_139_20_03',
 'EUV2003_139_21_00',
 'EUV2003_139_21_56',
 'EUV2003_139_22_52',
 'EUV2003_139_23_48']

cat.set_next_day()

pids.extend(list(cat.get_euv_date().query("OBSERVATION_TYPE=='CALIB'").index))

pids

['EUV2003_139_18_11',
 'EUV2003_139_19_07',
 'EUV2003_139_20_03',
 'EUV2003_139_21_00',
 'EUV2003_139_21_56',
 'EUV2003_139_22_52',
 'EUV2003_139_23_48',
 'EUV2003_140_00_44',
 'EUV2003_140_01_41',
 'EUV2003_140_02_37',
 'EUV2003_140_03_33',
 'EUV2003_140_04_29',
 'EUV2003_140_05_25',
 'EUV2003_140_06_22']

kwargs = {"x": "nx", "y": "ny", "cmap": "viridis", "clim": (0, 50)}

Column-averaging

class FlatFielder:
    def __init__(self, pid):
        self.pid = pid
        self.data = UVPDS(pid).xarray.astype("int16")

    @property
    def plot_set(self):
        return self.data.hvplot(
            x="spectral", y="spatial", cmap="viridis", title=self.pid
        )

    @property
    def integrated(self):
        return self.data.sum(dim="samples")

    @property
    def plot_integrated(self):
        return self.integrated.hvplot(
            x="spectral", y="spatial", cmap="viridis", title=self.pid
        )

    @property
    def averaged(self):
        return self.integrated.sel(spatial=slice(3, 61)).mean(dim="spatial")

    @property
    def plot_averaged(self):
        return self.averaged.hvplot(x="spectral", title=self.pid)

    @property
    def column_std(self):
        return self.integrated.sel(spatial=slice(2, 60)).std(dim="spatial")

    @property
    def plot_column_std(self):
        return self.column_std.hvplot(x="spectral", title=f"{self.pid}, Column STD")

    @property
    def ff(self):
        return self.integrated / self.averaged

    @property
    def plot_ff(self):
        return self.ff.hvplot(x="spectral", y="spatial", cmap="viridis", title=self.pid)

len(pids)

14

pids

['EUV2003_139_18_11',
 'EUV2003_139_19_07',
 'EUV2003_139_20_03',
 'EUV2003_139_21_00',
 'EUV2003_139_21_56',
 'EUV2003_139_22_52',
 'EUV2003_139_23_48',
 'EUV2003_140_00_44',
 'EUV2003_140_01_41',
 'EUV2003_140_02_37',
 'EUV2003_140_03_33',
 'EUV2003_140_04_29',
 'EUV2003_140_05_25',
 'EUV2003_140_06_22']

flatter = FlatFielder(pids[0])

flatter.plot_set

flatter.plot_integrated

flatter.plot_column_std

flatter.integrated.hvplot(x="spatial")

flatter.plot_ff

Steffl Calib class

Putting above together into a class

steffl = StefflCalib(pids)

pids

['EUV2003_139_18_11',
 'EUV2003_139_19_07',
 'EUV2003_139_20_03',
 'EUV2003_139_21_00',
 'EUV2003_139_21_56',
 'EUV2003_139_22_52',
 'EUV2003_139_23_48',
 'EUV2003_140_00_44',
 'EUV2003_140_01_41',
 'EUV2003_140_02_37',
 'EUV2003_140_03_33',
 'EUV2003_140_04_29',
 'EUV2003_140_05_25',
 'EUV2003_140_06_22']

for m in range(4):
    steffl.m = m
    for i in range(15, 998):
        steffl.i = i
        steffl.corrections[:, i, m] = steffl.get_averaged_triplet() / steffl.get_triplet_data()[0]

/home/maye/miniconda3/envs/py38/lib/python3.8/site-packages/xarray/core/computation.py:742: RuntimeWarning: divide by zero encountered in true_divide
  result_data = func(*input_data)
/home/maye/miniconda3/envs/py38/lib/python3.8/site-packages/xarray/core/computation.py:742: RuntimeWarning: invalid value encountered in true_divide
  result_data = func(*input_data)

steffl.plot_corrections(0)

steffl.plot_corrections(1)

steffl.corrections.shape

(64, 1024, 5)

steffl.plot_triplet()

steffl.get_averaged_triplet()

array([0.00000000e+00, 0.00000000e+00, 1.03666667e+03, 2.44133333e+03,
       2.29500000e+03, 1.98833333e+03, 1.78433333e+03, 2.38500000e+03,
       2.24466667e+03, 1.91800000e+03, 2.22333333e+03, 2.17300000e+03,
       1.81233333e+03, 1.92266667e+03, 2.09233333e+03, 2.54666667e+03,
       2.03866667e+03, 2.56266667e+03, 2.21300000e+03, 1.91233333e+03,
       2.12633333e+03, 1.71133333e+03, 2.68400000e+03, 1.84933333e+03,
       2.63100000e+03, 2.06766667e+03, 2.64400000e+03, 1.92766667e+03,
       2.36400000e+03, 1.99966667e+03, 1.75333333e+03, 2.37766667e+03,
       2.27800000e+03, 1.64400000e+03, 2.56766667e+03, 2.60333333e+03,
       1.71633333e+03, 2.01800000e+03, 2.49433333e+03, 1.96033333e+03,
       2.62033333e+03, 2.24366667e+03, 2.15033333e+03, 2.63866667e+03,
       2.43900000e+03, 2.57533333e+03, 2.66033333e+03, 1.85633333e+03,
       1.98533333e+03, 2.56100000e+03, 2.50233333e+03, 2.69100000e+03,
       2.25366667e+03, 2.29566667e+03, 2.81633333e+03, 2.42533333e+03,
       1.98266667e+03, 2.72366667e+03, 2.34466667e+03, 2.18833333e+03,
       2.51800000e+03, 2.73833333e+03, 2.07333333e+02, 1.33333333e+00])

steffl.plot()

class Col2Col:
    def __init__(self, pids, i=15, m=0):  # set of product_ids
        self.pids = pids
        self.i = i
        self.m = 0
        scan_df = pd.DataFrame({"pids": pids})
        scan_df.index.name = "m"

        stacked = []
        for m, pid in scan_df.iterrows():
            flatter = FlatFielder(pid.get(0))
            stacked.append(flatter.integrated)
        stack = np.dstack(stacked)
        arr = xr.DataArray(
            stack,
            dims=["spectral", "spatial", "scan"],
            coords={
                "scan": scan_df.index.values,
                "spectral": flatter.integrated.spectral,
                "spatial": flatter.integrated.spatial,
            },
        )
        arr.name = "scan_stack"
        self.arr = arr

    @property
    def triple_columns(self):
        return [self.i, self.i + 4, self.i + 8]

    @property
    def get_triplet(self, i=15, m=0):
        cols = []
        for col, scan in zip([0, 4, 8], [0, 5, 10]):
            cols.append(self.arr.isel(spectral=i + col, scan=m + scan))
        plots = []
        for col, col_number in zip(cols, [i, i + 4, i + 8]):
            plots.append(col.hvplot(label=f"Columns {col_number}, Scan {m}"))
        return hv.Overlay(plots)

col2col = Col2Col(pids)

col2col.triple_columns

[15, 19, 23]

get_triplet()

import panel as pn

pn.extension()

pn.interact(get_triplet)

def get_all_triplets(arr, i=0):
    triplets = []
    for m in range(4):
        plots = get_triplet(arr, i, m)
        triplets.append(hv.Overlay(plots))
    return triplets

triplets = get_all_triplets(arr, 15)

hv.Layout(triplets).cols(2)

col1 = arr.isel(spectral=i + 4, scan=m + 5)
col2 = arr.isel(spectral=i + 8, scan=m + 10)

(
    col0.hvplot(label=f"Scan {col0.scan.data}")
    * col1.hvplot(label=f"Scan {col1.scan.data}")
    * col2.hvplot(label=f"Scan {col2.scan.data}")
).opts(title=f"Column {i}")

((col0 + col1 + col2) / 3).hvplot()

col_set.mean(dim=["scan", "spectral"]).hvplot()

archive_df.loc["EUV2001_093_08_35_28"]

path    /home/maye/uvis_archive/observations/EUV2001_0...
det                                                   EUV
Name: EUV2001_093_08_35_28, dtype: object

import hvplot.xarray
import xarray as xr

ds = xr.open_dataset(fname)
ds

<xarray.Dataset>
Dimensions:   (integrations: 54, spatial_dim_0: 1, spectral_dim_0: 1024)
Dimensions without coordinates: integrations, spatial_dim_0, spectral_dim_0
Data variables:
    window_0  (integrations, spatial_dim_0, spectral_dim_0) int16 0 1 15 ... 0 0
Attributes: (12/16)
    windows:               1.0
    compression:           0
    odc_id:                7
    integration:           32
    channel:               EUV
    hvps_level:            0
    ...                    ...
    stop_time:             
    Version:               1
    start_time_str:        1999-007 17:05:02.000 (1999-Jan-07) SCClock=(12944...
    SCTime:                1294420183
    SCTimeFine:            0
    NetCDFWriter Version:  1.0

array([[[0, 1, ..., 1, 0]],

       [[0, 0, ..., 0, 0]],

       ...,

       [[1, 0, ..., 0, 0]],

       [[1, 0, ..., 0, 0]]], dtype=int16)

np.percentile(ds.window_0, (5, 95))

array([  0., 201.])

ds.window_0.hvplot.image(
    x="spectral_dim_0", y="integrations", cmap="viridis", clim=(0, 201)
)

ds.window_0.mean("integrations").hvplot(x="spectral_dim_0")

ds.window_0.plot()

<matplotlib.collections.QuadMesh at 0x7f01a1bb1790>

p = obsdir / "index_repaired.tab"

df = pd.read_csv(p, quotechar='"', skipinitialspace=True)

/home/maye/miniconda3/envs/py38/lib/python3.8/site-packages/IPython/core/interactiveshell.py:3441: DtypeWarning: Columns (0,9,10,11,12,13) have mixed types.Specify dtype option on import or set low_memory=False.
  exec(code_obj, self.user_global_ns, self.user_ns)

from planetarypy.pds.indexes import find_mixed_type_cols

find_mixed_type_cols(df, fix=False)

9
1999-01-07 10:05:02.093
1999-01-07 10:08:34.0
N/A
EUV1999-01-07 17:05:02.000
Unnamed: 6
USTARE
to evaluate EUV and FUV functions.
7
32
0
0.1
0.2

['9',
 '1999-01-07 10:05:02.093',
 '1999-01-07 10:08:34.0',
 'N/A',
 'EUV1999-01-07 17:05:02.000',
 'Unnamed: 6',
 'USTARE',
 'to evaluate EUV and FUV functions.',
 '7',
 '32',
 '0',
 '0.1',
 '0.2']

df.columns

Index(['9', '1999-01-07 10:05:02.093', '1999-01-07 10:08:34.0', 'EUV', 'N/A',
       'EUV1999-01-07 17:05:02.000', 'Unnamed: 6', 'USTARE',
       'to evaluate EUV and FUV functions.', '7', '32', '0', '0.1', '0.2',
       '0.3', '1', '0.4'],
      dtype='object')

index.columns

Index(['FILE_NAME', 'OBSERVATION_TYPE', 'START_TIME', 'STOP_TIME',
       'TARGET_NAME', 'DATA_SET_ID', 'SPACECRAFT_CLOCK_START_COUNT',
       'SPACECRAFT_CLOCK_STOP_COUNT', 'INTEGRATION_DURATION',
       'COMPRESSION_TYPE', 'HI_VOLTAGE_POWER_SUPPLY_STATE',
       'OCCULTATION_PORT_STATE', 'SLIT_STATE', 'TEST_PULSE_STATE', 'ODC_ID',
       'RIGHT_ASCENSION', 'DECLINATION', 'SUB_SOLAR_LATITUDE',
       'SUB_SOLAR_LONGITUDE', 'SUB_SPACECRAFT_LATITUDE',
       'SUB_SPACECRAFT_LONGITUDE', 'PHASE_ANGLE', 'EMISSION_ANGLE',
       'SOLAR_INCIDENCE_ANGLE', 'CENTRAL_BODY_DISTANCE', 'DWELL_TIME',
       'H_LEVEL', 'D_LEVEL', 'filename'],
      dtype='object')

index[index.filename.str.startswith("EUV")].iloc[0]

FILE_NAME                        /COUVIS_0001/DATA/D1999_007/EUV1999_007_17_05.LBL
OBSERVATION_TYPE                                                            USTARE
START_TIME                                              1999-01-07 17:05:01.949000
STOP_TIME                                               1999-01-07 17:08:37.949000
TARGET_NAME                                                                    NaN
DATA_SET_ID                                                  CO-J-UVIS-2-SPEC-V1.2
SPACECRAFT_CLOCK_START_COUNT                                      1/1294420183.000
SPACECRAFT_CLOCK_STOP_COUNT                                                    UNK
INTEGRATION_DURATION                                                           4.0
COMPRESSION_TYPE                                                              NONE
HI_VOLTAGE_POWER_SUPPLY_STATE                                                  OFF
OCCULTATION_PORT_STATE                                                      CLOSED
SLIT_STATE                                                         HIGH_RESOLUTION
TEST_PULSE_STATE                                                                ON
ODC_ID                                                                           7
RIGHT_ASCENSION                                                             -999.0
DECLINATION                                                                 -999.0
SUB_SOLAR_LATITUDE                                                          -999.0
SUB_SOLAR_LONGITUDE                                                         -999.0
SUB_SPACECRAFT_LATITUDE                                                     -999.0
SUB_SPACECRAFT_LONGITUDE                                                    -999.0
PHASE_ANGLE                                                                 -999.0
EMISSION_ANGLE                                                              -999.0
SOLAR_INCIDENCE_ANGLE                                                       -999.0
CENTRAL_BODY_DISTANCE                                                       -999.0
DWELL_TIME                                           1969-12-31 23:59:59.999999001
H_LEVEL                                                                        NaN
D_LEVEL                                                                        NaN
filename                                                         EUV1999_007_17_05
Name: 9, dtype: object

obs.head()

cols = ["index start_time stop_time detector target obsid_time unknown type comment "]

	filename	slit	nx	ny	nz	int	odcid	x2	y2	name
0	EUV1999_016_19_47_15	2	1024	64	2	60	12	1023	63	alp vir
1	EUV1999_016_19_49_06	2	1024	1	123	1	13	1023	63	alp vir
2	EUV1999_016_20_08_10	2	1	64	399	1	11	1023	63	alp vir
3	EUV1999_016_20_16_14	2	1024	64	2	60	12	1023	63	alp vir
4	EUV1999_016_20_18_06	2	1024	1	123	1	13	1023	63	alp vir

Column-averaging

Steffl Calib class

class StefflCalib[source]

`class` `StefflCalib`[source]